{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. 库的引用和数据导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import collections\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(file_name):\n",
    "    with open(file_name) as file:\n",
    "        temp_title = next(file)[:-1].split(',')\n",
    "        Sample = collections.namedtuple(\"Samples\", temp_title)\n",
    "        samples = list()\n",
    "        for raw_sample in file:\n",
    "            sample = raw_sample[:-1].split(',')\n",
    "            samples.append(Sample._make(sample))\n",
    "    return temp_title, samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "gaze_file = \"gaze.csv\"\n",
    "all_titles, origin_data = load_data(gaze_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 通过信度筛选数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def choose_confidences(data):\n",
    "    def confidence_judge(sample):\n",
    "        return float(sample.confidence) >= 0.9\n",
    "    return tuple(filter(confidence_judge, data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "可信数据条数: 87442\n"
     ]
    }
   ],
   "source": [
    "confident_data = choose_confidences(origin_data)\n",
    "print(f\"可信数据条数: {len(confident_data)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 通过3 sigma原则筛选数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_mean(data):\n",
    "    return sum(data) / len(data)\n",
    "\n",
    "\n",
    "def my_std(data, mean):\n",
    "    square = tuple(map(lambda x: (x - mean) ** 2, data))\n",
    "    return math.sqrt(sum(square) / len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_by_3sigma(data):\n",
    "    x_cluster = tuple(map(lambda x: float(x.norm_pos_x), data))\n",
    "    y_cluster = tuple(map(lambda y: float(y.norm_pos_y), data))\n",
    "    x_mean = my_mean(x_cluster)\n",
    "    y_mean = my_mean(y_cluster)\n",
    "    x_std = my_std(x_cluster, x_mean)\n",
    "    y_std = my_std(y_cluster, y_mean)\n",
    "    x_max = x_mean + 3 * x_std\n",
    "    x_min = x_mean - 3 * x_std\n",
    "    y_max = y_mean + 3 * y_std\n",
    "    y_min = y_mean - 3 * y_std\n",
    "\n",
    "    def sigma_judge(sample):\n",
    "        x_correct = x_min < float(sample.norm_pos_x) < x_max\n",
    "        y_correct = y_min < float(sample.norm_pos_y) < y_max\n",
    "        return x_correct and y_correct\n",
    "    return tuple(filter(sigma_judge, data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "有效数据条数: 86185\n"
     ]
    }
   ],
   "source": [
    "sigma_left_data = delete_by_3sigma(confident_data)\n",
    "print(f\"有效数据条数: {len(sigma_left_data)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. 根据原始样本估计平均采样率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_rate(data):\n",
    "    time_stamp = tuple(map(lambda x: float(x.gaze_timestamp) - 88150, data))\n",
    "    return 1 / (my_mean(time_stamp[1:]) - my_mean(time_stamp[:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "设备采样率: 427.623Hz\n"
     ]
    }
   ],
   "source": [
    "rate = calculate_rate(origin_data)\n",
    "print(f\"设备采样率: {round(rate, 3)}Hz\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. 重采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resample(data):\n",
    "    mark = float(data[0].gaze_timestamp)\n",
    "    nearest, result, cycle = data[0], list(), 1 / 100\n",
    "\n",
    "    def distance(sample):\n",
    "        return abs(float(sample.gaze_timestamp) - mark)\n",
    "    for item in data:\n",
    "        if distance(item) > distance(nearest):\n",
    "            result.append(nearest)\n",
    "            while mark < float(item.gaze_timestamp):\n",
    "                mark += cycle\n",
    "        nearest = item\n",
    "    return tuple(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88150.8256385, 2, 0.9151973420751922, 0.4936116815701956, 0.39895021008258913, 88150.827729-0 88150.823548-1, 2.4128100758327182, -21.468145193186366, -295.0710570327717, 20.0, 15.0, -20.0, 0.06288346587545898, 0.02509974236087054, 0.9977052032804625, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.15004148584820762, 0.23453606423953438, 0.9604584254903964\n",
      "88150.82966349999, 2, 0.9151973420751922, 0.49356951516031494, 0.3995420469713393, 88150.827729-0 88150.83159799999-1, 2.427118214715252, -21.32819300553827, -294.87451607757225, 20.0, 15.0, -20.0, 0.06288346587545898, 0.02509974236087054, 0.9977052032804625, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.1502104716193903, 0.23374654728633698, 0.9606244666088807\n",
      "88150.87752149999, 4, 0.94500047108518, 0.4920943029105782, 0.3989300839628567, 88150.879586-0 88150.87545699999-1, 3.018227170856065, -21.704819872309287, -298.2646502338695, 20.0, 15.0, -20.0, 0.06004036186469085, 0.024936294324659092, 0.9978844302686117, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.1504509230615827, 0.2334355900843485, 0.9606624511408207\n",
      "88150.8860805, 4, 0.9336578810932825, 0.49216591828673045, 0.3991673717146227, 88150.88389-0 88150.88827099999-1, 2.993379604392894, -21.671916187220244, -298.5133295164393, 20.0, 15.0, -20.0, 0.06007445660943474, 0.024427210448378187, 0.997894969950642, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.1502346770715741, 0.23348674078640194, 0.9606838625074033\n",
      "88150.8917005, 4, 0.923014300448957, 0.4917446414950056, 0.3979021415944922, 88150.891699-0 88150.891702-1, 3.1948566002857905, -22.225656140802766, -302.3468596140583, 20.0, 15.0, -20.0, 0.05856004760502819, 0.024550784512162607, 0.9979819536466253, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14891582435890977, 0.23373323204977325, 0.9608292530367055\n",
      "88150.9141995, 5, 0.9291327620219251, 0.4914388450903873, 0.39748146803382634, 88150.91139099999-0 88150.91700799999-1, 3.3142014956882235, -22.32397147411286, -302.4381576835125, 20.0, 15.0, -20.0, 0.05812076850806207, 0.02427722241941762, 0.9980143249170481, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14925633872709826, 0.2345754008162153, 0.9605711460801297\n",
      "88150.9361805, 5, 0.9976716979910984, 0.49109736711588986, 0.3962520166514687, 88150.93622799999-0 88150.936133-1, 3.4655296294769027, -22.717123650092198, -304.1173390246404, 20.0, 15.0, -20.0, 0.05725573600184664, 0.02556757341574248, 0.9980321036342059, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14890087750610775, 0.23447839734970133, 0.9606499934181163\n",
      "88150.945604, 6, 0.9147837667828747, 0.4913582478042822, 0.3960199736965523, 88150.94368499999-0 88150.947523-1, 3.374399071467657, -22.838474090435426, -305.05957760399974, 20.0, 15.0, -20.0, 0.05738168673496691, 0.025708572282394638, 0.9980212479394674, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14811747558506844, 0.23433106058094225, 0.9608070396668197\n",
      "88151.015644, 8, 0.9128100556440778, 0.4906790676154535, 0.39639173999521227, 88151.01567699999-0 88151.015611-1, 3.6281507796660044, -22.68521380338881, -304.099705874219, 20.0, 15.0, -20.0, 0.05670169961397593, 0.025972079770318734, 0.9980532893254206, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14946539700782105, 0.23388052031508275, 0.9607080708073815\n",
      "88151.025782, 8, 0.9371902869430557, 0.4906599955259159, 0.3973699924620683, 88151.025782-0 88151.025782-1, 3.6507555670512577, -22.564855640909784, -305.36953110383615, 20.0, 15.0, -20.0, 0.05636936830060078, 0.023263743027513203, 0.9981389144691944, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14888219028511426, 0.23460608817767314, 0.9606217136864422\n",
      "88151.02694049999, 8, 0.9371902869430557, 0.4904516632242954, 0.398487455597748, 88151.025782-0 88151.02809899999-1, 3.7183020101987196, -22.236128402522628, -304.23345067375914, 20.0, 15.0, -20.0, 0.05636936830060078, 0.023263743027513203, 0.9981389144691944, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14971249152618837, 0.2333930499290845, 0.9607881421654937\n",
      "88151.125409, 11, 0.9923533549628162, 0.4759974615098218, 0.3890322538481459, 88151.12743299999-0 88151.123385-1, 10.944627738690539, -28.461849092999916, -356.2327553125614, 20.0, 15.0, -20.0, 0.026569346325971726, 0.03411085734139814, 0.9990648223449999, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14777132279127167, 0.2196996760440456, 0.9643109915927844\n",
      "88151.135373, 11, 0.9940691187646469, 0.45735842840979035, 0.4068041987736185, 88151.13536599999-0 88151.13537999999-1, 26.37437430241233, -32.42409296791041, -483.21342660106075, 20.0, 15.0, -20.0, -0.013670645400479136, 0.04384029635353002, 0.9989450144377168, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.14076232387968576, 0.1584982315862648, 0.9772739016058994\n",
      "88151.165396, 12, 0.9928466733441283, 0.41733807236928716, 0.42538663806396815, 88151.163422-0 88151.16737-1, 262.30341479101065, -133.17955260086075, -2479.0680386860213, 20.0, 15.0, -20.0, -0.09788374320242656, 0.06139750181587196, 0.993302129056136, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.1217757118897545, 0.058165471798393965, 0.9908518829190436\n",
      "88151.175385, 12, 0.9663552612170796, 0.4128119758146548, 0.419429254223597, 88151.171403-0 88151.17936699999-1, 391.0722434429901, -203.28253464965258, -3504.2105041897444, 20.0, 15.0, -20.0, -0.10569701861648621, 0.06407430458522569, 0.9923319120876376, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.12253306971297351, 0.06015775439435803, 0.990639536568647\n",
      "88151.185692, 13, 0.9294503554799421, 0.41160516675266134, 0.41747531018383255, 88151.18770299999-0 88151.183681-1, 464.2275307170954, -243.7869415853015, -4102.929380022897, 20.0, 15.0, -20.0, -0.10794843650259221, 0.06501392866812675, 0.992028388775131, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.12231010572726742, 0.06065333048196549, 0.9906368716832775\n",
      "88151.197391, 13, 0.9178147883752031, 0.4119853597521403, 0.4169264931465607, 88151.199373-0 88151.19540899999-1, 459.71402446899333, -244.0719695135335, -4080.589099779168, 20.0, 15.0, -20.0, -0.10744288301291322, 0.06383842360118842, 0.9921596053871523, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.12188540571259907, 0.06266313203321537, 0.9905641219820467\n",
      "88151.205431, 13, 0.9070761366591551, 0.41193483629834676, 0.41509305143188635, 88151.203474-0 88151.207388-1, 469.7716823276799, -254.77009405751667, -4167.4722602214415, 20.0, 15.0, -20.0, -0.10758788340517399, 0.06537189529453102, 0.9920440326164944, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.12172680550025683, 0.06358390623264314, 0.9905249475358514\n",
      "88151.21535749998, 14, 0.9899267474818387, 0.41236226094544914, 0.41650600867041965, 88151.21537399999-0 88151.21534099999-1, 445.0920280780266, -238.5263707501626, -3967.7900261611258, 20.0, 15.0, -20.0, -0.10684200022375505, 0.0642285009431517, 0.9921993180076187, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.12169879343256274, 0.0631045780272922, 0.9905590421116027\n",
      "88151.217814, 14, 0.997836588046421, 0.41264819754020554, 0.41593804447731264, 88151.21537399999-0 88151.220254-1, 482.9967796226188, -261.4534517077724, -4319.787611181237, 20.0, 15.0, -20.0, -0.10684200022375505, 0.0642285009431517, 0.9921993180076187, -39.93492801030824, 14.997919452545707, -20.075282531198983, -0.12048539967153077, 0.06326183290716106, 0.9906973346906804\n"
     ]
    }
   ],
   "source": [
    "smaller = resample(sigma_left_data)\n",
    "for item in smaller[:20]:\n",
    "    print(', '.join([x for x in item]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 更改时间格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def edit_time(data):\n",
    "    def replace_time(sample):\n",
    "        time_stamp = float(sample.gaze_timestamp)\n",
    "        new_time = str(datetime.fromtimestamp(time_stamp))\n",
    "        return sample._replace(gaze_timestamp=new_time)\n",
    "    result = [replace_time(x) for x in data]\n",
    "    return tuple(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1970-01-02 08:29:10.825638\n",
      "1970-01-02 08:29:10.829663\n",
      "1970-01-02 08:29:10.877521\n",
      "1970-01-02 08:29:10.886081\n",
      "1970-01-02 08:29:10.891700\n"
     ]
    }
   ],
   "source": [
    "new_time = edit_time(smaller)\n",
    "for item in new_time[:5]:\n",
    "    print(item.gaze_timestamp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. 数据写回"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_back(titles, data):\n",
    "    result = ','.join(titles) + '\\n'\n",
    "    for item in data:\n",
    "        temp = ','.join(item) + '\\n'\n",
    "        result += temp\n",
    "    with open(\"gaze.csv\", \"w\") as file:\n",
    "        file.write(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_back(all_titles, new_time)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
